/**
  ******************************************************************************
  * @file    main.c
  * @author  Dragon.Chen
  * @version V2.0.0
  * @date    09-22-2016
  * @brief   主程序首先解析配置文件，然后根据命令行参数决定程序执行流程
  *          -help:
  *             打印程序帮助信息;
  *          —version:
  *             查看服务器程序版本号;
  *          -config:
  *             指定服务器配置文件路径;
  *          -reconfig
  *             重新读配置文件
  *          -ca
  *             重读ca
  *          -crl
  *             重读crl
  *          -stop:
  *             关闭服务器
  *          -remove:
  *             强迫用户下线数据格式为:"aa;bb;cc;dd;ee;",最后一个分号不可以
  *          省略，否则无法使最后一个用户下线;
  *          -start：
  *             启动服务器
  ******************************************************************************
  * @attention 不要轻易改动此文件，除非你知道自己在做啥！！！
  *
  * <h2><center>&copy; COPYRIGHT 2016 上海今中网络科技有限公司 </center></h2>
  ******************************************************************************
  */

#include <getopt.h>
#include <sys/shm.h>
#include <sys/wait.h>
#include <sys/time.h>
#include <sys/signal.h>
#include "backwork.h"

/// 函数申明
static void create_worker_process_pool();

/**
  * @brief  程序退出清理
  * @param  void
  * @retval void
  */
static void cleanup_before_master_exit()
{
    if (StopWorkingWorker)
    {
        free(StopWorkingWorker);
        StopWorkingWorker = NULL;
    }

    if (BusyRatioPtr)
    {
        free(BusyRatioPtr);
        BusyRatioPtr = NULL;
    }

    if (WorkerProcess)
    {
        free(WorkerProcess);
        WorkerProcess = NULL;
    }

    if (WorkerProcessIndex == -1)
    {
        close_log();
        /* 删除shmid，并且拆除共享内存区域,由master进程完成*/
        if (shmctl(ShmLogoffShmId,  IPC_RMID, NULL) != 0)
            GetLastErrorMsg(LOG_ERROR, "ShmLogoffShmId");
        if (shmctl(ShmBusyRatioShmId, IPC_RMID, NULL) != 0)
            GetLastErrorMsg(LOG_ERROR, "ShmBusyRatioShmId");
        if (semctl(ShmLogoffSemId,  0, IPC_RMID, 0) != 0)
            GetLastErrorMsg(LOG_ERROR, "ShmLogoffSemId");
        if (semctl(ShmBusyRatioSemId, 0, IPC_RMID, 0) != 0)
            GetLastErrorMsg(LOG_ERROR, "ShmBusyRatioSemId");
    }
    else
    {
        /* 段接这个共享内存区，程序终止时，也会自动段接这个共享内存区 */
        shmdt(ShmLogoff);
        shmdt(ShmBusyRatio);
    }
}

/**
  * @brief  信号处理函数(实现统一信号事件源)
  * @param  arg1:信号
  * @retval void
  */
static void signal_handler(int signal)
{
    int saveErrno = errno;
    unsigned char msg = (unsigned char)signal;

    send(SigPipefd[1], &msg, 1, 0);
    errno = saveErrno;
}

/**
  * @brief  添加信号处理函数
  * @param  arg1:信号  arg2:函数指针
  * @retval void
  */
static void add_signal(int signal, void (handler)(int))
{
    struct sigaction sa;
    ZeroMemory(&sa, sizeof(sa));

    sa.sa_handler = handler;
    sa.sa_flags |= SA_RESTART;
    sigfillset(&sa.sa_mask);
    assert_param(sigaction(signal, &sa, NULL) != -1);
}

/**
  * @brief  创建事件表描述符和信号管道
  * @param  void
  * @retval void
  */
static void init_worker_signal_pipe()
{
    CLIENT_DATA_T *clientData;
    assert_param((Epollfd = epoll_create(5)) != -1 );
    assert_param(socketpair(PF_UNIX, SOCK_STREAM, 0, SigPipefd) != -1 );

    /// 用于信号产生时，发送信号到主循环,主循环中接收信号并处里该信号
    set_nonblocking(SigPipefd[1]);
    clientData = get_client_data(&ClientDataManage);
    assert_param(clientData != NULL);
    clientData->sockfd = SigPipefd[0];
    add_read_event(Epollfd, clientData, ET);

    /// 信号处理方式
    add_signal(SIGALRM, signal_handler);
    add_signal(SIGCHLD, signal_handler);
    add_signal(SIGTERM, signal_handler);
    add_signal(SIGINT,  signal_handler);
    add_signal(SIGPIPE, SIG_IGN);
}

/**
  * @brief  创建事件表描述符和信号管道
  * @param  void
  * @retval void
  */
static void init_master_signal_pipe()
{
    assert_param((Epollfd = epoll_create(5)) != -1);
    assert_param(socketpair(PF_UNIX, SOCK_STREAM, 0, SigPipefd) != -1);

    /// 用于信号产生时，发送信号到主循环,主循环中接收信号并处里该信号
    set_nonblocking(SigPipefd[1]);
    add_read_fd(Epollfd, SigPipefd[0], ET);

    /// 信号处理方式
    add_signal(SIGALRM, signal_handler);
    add_signal(SIGCHLD, signal_handler);
    add_signal(SIGTERM, signal_handler);
    add_signal(SIGUSR2, signal_handler);
    add_signal(SIGINT,  signal_handler);
    add_signal(SIGPIPE, SIG_IGN);
}

/**
  * @brief  master进程获取最不繁忙的worker进程
  * @param  void
  * @retval 最不繁忙的进程ID
  */
static inline int get_most_free_service()
{
    int index, minIndex = 0;
    int busyRatio = WorkerProcess[0].busyRatio;

    for (index=0; index<NWorkerProcess; index++)
    {
        if ((WorkerProcess[index].pid != -1) && (WorkerProcess[index].busyRatio < busyRatio))
        {
            minIndex = index;
            busyRatio = WorkerProcess[index].busyRatio;
        }
    }

    return minIndex;
}

/**
  * @brief  初始化其他数据
  * @param  arg1:delay second
  * @retval void
  */
static void init_other(int second)
{
    pid_t    pid;
    int      i, j   = 0;
    int      rows   = -1;
    int      cols   = -1;
    int     *pids   = NULL;
    BIO     *bio    = NULL;
    FILE    *fp     = NULL;
    X509    *cacert = NULL;
    char    *errmsg = NULL;
    char     sqlBuffer[SQL_BUFLEN] = {0};

    sleep(second);
    assert_param((fp = fopen(SERVER_BASEPATH SERVER_WORKER_PID, "rt")) != NULL);
    pids = (int *)calloc(NWorkerProcess, sizeof(int));
    assert_param(pids != NULL);

    /// get all valide process number
    while (fgets(sqlBuffer, SQL_BUFLEN, fp))
    {
        pid = atoi(sqlBuffer);
        if (kill(pid, 0) == 0)
            pids[j++] = pid;
        ZeroMemory(sqlBuffer, SQL_BUFLEN);
    }
    fclose(fp);

    /// open the sqlite3 database and set it timeout
    ZeroMemory(sqlBuffer, SQL_BUFLEN);
    if (OpenDatabase(ServerConfig.databasePath, dbHandle) != SQLITE_OK)
        MSG(LOG_FATAL, "Fail to open the database!");
    sqlite3_busy_timeout(dbHandle, ServerConfig.databaseTimeout);
    ///****************patched by yagamishi add*************2016/02/18*******
    ///******************发送日志********************************************
    if (OpenDatabase(ServerConfig.logDatabasePath, ZDLogDbHandle) != SQLITE_OK)
        MSG(LOG_FATAL, "Fail to open the zdjgLog database!");
    sqlite3_busy_timeout(ZDLogDbHandle, ServerConfig.databaseTimeout);
    ///**********************************************************************

    /// set client loginstate flag zero
    snprintf(sqlBuffer, SQL_BUFLEN, "UPDATE MPSInfo SET processId = 0, loginState = 0 "
                                    "WHERE processId NOT IN (");
    for (i=0; i<j; i++)
        snprintf(sqlBuffer+strlen(sqlBuffer), SQL_BUFLEN-strlen(sqlBuffer), "%d, ", pids[i]);
    sqlBuffer[strlen(sqlBuffer)-2] = ')';
    sqlBuffer[strlen(sqlBuffer)-1] = '\0';
    free(pids);
    pids = NULL;
    if (UpdateDataFromDatabase(dbHandle, sqlBuffer, errmsg) != SQLITE_OK)
        MSG(LOG_FATAL, "Set client loginstate flag error[%d] %s!", WorkerProcess[WorkerProcessIndex].pid, errmsg);

    /// get all name of CA path
    snprintf(sqlBuffer, SQL_BUFLEN, "SELECT CAFilter_path FROM tb_CAFilter");
    if (QueryDataFromDatabase(dbHandle, sqlBuffer, ServerConfig.caCertificatePath, rows, cols, errmsg) != SQLITE_OK)
        MSG(LOG_FATAL, "Query all name of CA path error %s!", errmsg);
    if (rows == 0)
        MSG(LOG_FATAL, "There is no certificate in the table of database!");
    ServerConfig.nCaCertificate = rows;

    /// import all ca certificate into X509 store
    # if OPENSSL_API_COMPAT < 0x10100000L
    //the fun in higher verison api is deprecated
    OpenSSL_add_all_algorithms(); /* absolutely important */
    # endif
    if ((RootCertStore = X509_STORE_new()) == NULL)
        MSG(LOG_FATAL, "Fail to open the CertStore!");

    for (i=0; i<ServerConfig.nCaCertificate; i++)
    {
        ZeroMemory(sqlBuffer, SQL_BUFLEN);
        snprintf(sqlBuffer, SQL_BUFLEN, "%s%s.cer", "/sonicom/conf/openvpn/keys/", ServerConfig.caCertificatePath[i+1]);

        if ((bio = BIO_new_file(sqlBuffer, "rt")) == NULL)
        {
            MSG(LOG_ERROR, "Certificate path error!");
            continue;
        }

        if ((cacert = PEM_read_bio_X509(bio, NULL, NULL, NULL)) == NULL )
        {
            BIO_free(bio);
            MSG(LOG_ERROR, "Parse the certification error!");
            continue;
        }

        X509_STORE_add_cert(RootCertStore, cacert);
        BIO_free(bio);
        X509_free(cacert);
    }
    FreeQueryDataFromDatabase(ServerConfig.caCertificatePath);
    ServerConfig.caCertificatePath = NULL;

    /// open certificate revoked list
    assert_param((bio = BIO_new_file(ServerConfig.certificateRevokeListPath, "r")) != NULL);
    assert_param((CertRevokedList = PEM_read_bio_X509_CRL(bio, NULL, NULL, NULL)) != NULL);
    BIO_free(bio);

    /// allocate vpn port according to nCPU
    VPNPortCount = (int *)calloc(ServerConfig.nCPU, sizeof(int));
    assert_param(VPNPortCount);
}

/**
  * @brief  执行worker进程循环
  * @param  NULL
  * @retval void
  */
static void run_worker_process_cycle()
{
    time_t              expireTime;
    int                 i, j, retval, number;
    int                 sockfd, connfd, timeout, fullFlag = 0;
    struct epoll_event  events[WORKER_MAX_EVENT_NUMBER];
    struct sockaddr_in  clientAddress;
    unsigned char       msg;
    unsigned char       sockBuf[WORKER_SOCK_BUFLEN];
    BOOL                stopWorkerCycle = FALSE;
    CLIENT_DATA_T      *clientData;
    socklen_t           clientAddressLength = sizeof(clientAddress);
    ZeroMemory(events, sizeof(events));

    init_other(2);
    InitSockBufferManage();
    InitClientDataManage();
    InitCertificateManage();
    init_worker_signal_pipe();
    init_timer(WORKER_MAX_NCLIENT, close_socket);

    clientData = get_client_data(&ClientDataManage);
    assert_param(clientData != NULL);
    clientData->sockfd = WorkerProcess[WorkerProcessIndex].pipefd[1];
    add_read_event(Epollfd, clientData, ET);
    set_nonblocking(Listenfd);

    MSG(LOG_INFO, "Worker[%d] starts working cycle now!", WorkerProcess[WorkerProcessIndex].pid);

    expireTime = get_min_timer_value();
    timeout = (expireTime - time(NULL)) * 1000;
    while (!stopWorkerCycle)
    {
        number = epoll_wait(Epollfd, events, WORKER_MAX_EVENT_NUMBER, timeout);

        if ((number < 0) && (errno != EINTR))
        {
            GetLastSocketErrorMsg(LOG_ERROR, "Worker epoll_wait error msg:");
            break;
        }

        if (number == 0)
        {
            worker_timer_timeout();
            expireTime = get_min_timer_value();
            timeout = (expireTime - time(NULL)) * 1000;
            continue;
        }

        /// 处理准备好的事件
        for (i=0; i<number; i++)
        {
            /// 检查事件是否过期(此处的clientData只对用户事件有用)
            if ( ((clientData = (CLIENT_DATA_T *)events[i].data.ptr) == NULL) ||
                 ((sockfd = clientData->sockfd) <= 0) )
                continue;

            /// 父子通讯管道可读
            if ((sockfd == WorkerProcess[WorkerProcessIndex].pipefd[1]) && (events[i].events & EPOLLIN))
            {
                while (TRUE)
                {
                    retval = recv(sockfd, sockBuf, WORKER_SOCK_BUFLEN, 0);
                    if (retval <= 0)
                        break;

                    for (j=0; j<retval; j++)
                    {
                        switch (sockBuf[j])
                        {
                            case MW_SOCK_ACCEPT :
                            {
                                while (TRUE)
                                {
                                    ZeroMemory(&clientAddress, sizeof(clientAddress));
                                    connfd = accept(Listenfd, (struct sockaddr *)&clientAddress, &clientAddressLength);
                                    if (connfd < 0)
                                    {
                                        break;  // 说明没用户链接了
                                    }

                                    /// 初始化用户数据
                                    clientData = get_client_data(&ClientDataManage);
                                    if (clientData == NULL)
                                    {
                                        close(connfd);
                                        if (fullFlag == 0)
                                        {
                                            MSG(LOG_WARNING, "Worker is full, can not accept new request!");
                                            fullFlag = 1;
                                        }
                                        continue;
                                    }

                                    fullFlag = 0;
                                    clientData->sockfd = connfd;
                                    clientData->ipAddress = (u_int)clientAddress.sin_addr.s_addr;
                                    clientData->callback = recv_welcome_request_cb;
                                    clientData->timer.expire = time(NULL) + WORKER_LOGIN_TIMEOUT;
                                    clientData->timer.clientData = clientData;
                                    add_read_event(Epollfd, clientData, ET);
                                    add_timer(&clientData->timer); /* 虽然有返回值，但不会添加失败 */

                                    /// 更新当前进程用户的数量
                                    WorkerProcess[WorkerProcessIndex].busyRatio++;
                                }

                                ShmBusyRatioLock();
                                snprintf(BusyRatioPtr[WorkerProcessIndex], SERVER_SHM_CT_PER_SIZE - 1, "%d", WorkerProcess[WorkerProcessIndex].busyRatio);
                                ShmBusyRatioUnlock();

                                msg = MW_BUSY_RATIO;
                                send(WorkerProcess[WorkerProcessIndex].pipefd[1], &msg, 1, 0); // 不关心发送成功与否

                                break;
                            }

                            case MW_READ_SHM_LOGOFF :
                            {
                                worker_remove_client();
                                break;
                            }

                            case MW_RECONFIG :
                            {
                                worker_reread_config();
                                break;
                            }

                            case MW_RELOAD_CA :
                            {
                                worker_reload_ca();
                                break;
                            }

                            case MW_RELOAD_CRL :
                            {
                                worker_read_crl();
                                break;
                            }

                            default : break;
                        } // end of switch
                    } // end of for

                    if (retval < WORKER_SOCK_BUFLEN)
                        break;
                } //end of while
            }
            /// 信号管道
            else if ( (sockfd == SigPipefd[0]) && (events[i].events & EPOLLIN) )
            {
                while (TRUE)
                {
                    retval = recv(sockfd, sockBuf, WORKER_SOCK_BUFLEN, 0);
                    if (retval <= 0)
                        break;

                    for (j=0; j<retval; j++)
                    {
                        switch (sockBuf[j])
                        {
                            case SIGCHLD :
                                break;

                            case SIGTERM :
                            case SIGINT :
                            {
                                MSG(LOG_INFO, "Worker[%d] will exit!", WorkerProcess[WorkerProcessIndex].pid);
                                stopWorkerCycle = TRUE;
                                break;
                            }

                            default :
                                break;
                        } // end of switch
                    } // end of for

                    if (retval < WORKER_SOCK_BUFLEN)
                        break;
                } // end of while
            }
            /// 读/写client数据
            else if ( (events[i].events & EPOLLIN) || (events[i].events & EPOLLOUT) )
            {
                if ( clientData->callback(clientData) == RETURN_FAILURE )
                {
                    free_client_data(&ClientDataManage, clientData);

                    if (WorkerProcess[WorkerProcessIndex].busyRatio > 0)     // 更新当前进程用户的数量
                        WorkerProcess[WorkerProcessIndex].busyRatio--;

                    ShmBusyRatioLock();
                    snprintf(BusyRatioPtr[WorkerProcessIndex], SERVER_SHM_CT_PER_SIZE - 1, "%d", WorkerProcess[WorkerProcessIndex].busyRatio);
                    ShmBusyRatioUnlock();

                    msg = MW_BUSY_RATIO;
                    send(WorkerProcess[WorkerProcessIndex].pipefd[1], &msg, 1, 0); // 不关心发送成功与否
                }
            }
            else
            {}

        } // end of for

        /// 检查是否超时
        timeout = (expireTime - time(NULL)) * 1000;
        if (timeout <= 0)
            worker_timer_timeout();
        expireTime = get_min_timer_value();
        timeout = (expireTime - time(NULL)) * 1000;

    } // end of while

    finalize_timer();
    finalize_sock_buffer(&SockBufferManage);
    finalize_certificate(&CertificateManage);
    finalize_client_data(&ClientDataManage);
    FinalizeOther();        ///此处应使用宏函数命名规则

    close_fd(Epollfd, WorkerProcess[WorkerProcessIndex].pipefd[1]);
    close_fd(Epollfd, SigPipefd[0]);
    close(SigPipefd[1]);
    close(Listenfd);
    close(Epollfd);

    MSG(LOG_INFO, "Worker[%d] stops working normally now!", WorkerProcess[WorkerProcessIndex].pid);
}

/**
  * @brief  处理master的user2信号
  * @param  worker进程索引
  * @retval master进程返回-1，worker进程返回0
  */
static void handle_master_signal_user2()
{
    int            k, sum;
    u_char         msg;
    FILE          *fp = NULL;
    SIGNAL_USER_T  signalUser;

    ZeroMemory(&signalUser, sizeof(SIGNAL_USER_T));
    if ((fp=fopen(SERVER_BASEPATH SERVER_SIGNAL_PATH, "rb")) == NULL)
        return;
    fread(&signalUser, sizeof(SIGNAL_USER_T), 1, fp);
    fclose(fp);

    /// 剔除用户
    if (signalUser.remove)
    {
        for (sum = 0, k=0; k<NWorkerProcess; k++)
        {
            if (WorkerProcess[k].pid != -1)
                sum++;
        }

        msg = MW_READ_SHM_LOGOFF;
        snprintf(ShmLogoffHead, SERVER_SHM_LF_HEAD_SIZE - 1, "%d", sum);

        for (k=0; k<NWorkerProcess; k++)
        {
            if (WorkerProcess[k].pid != -1)
                send(WorkerProcess[k].pipefd[0], &msg, 1, 0);  // 发送失败也没办法
        }

        return;
    }

    /// 重读配置文件
    if (signalUser.reconfig)
    {
        msg = MW_RECONFIG;
        for (k=0; k<NWorkerProcess; k++)
        {
            if (WorkerProcess[k].pid != -1)
                send(WorkerProcess[k].pipefd[0], &msg, 1, 0);
        }
    }

    /// 重新载入CA
    if (signalUser.reloadCA)
    {
        msg = MW_RELOAD_CA;
        for (k=0; k<NWorkerProcess; k++)
        {
            if (WorkerProcess[k].pid != -1)
                send(WorkerProcess[k].pipefd[0], &msg, 1, 0);
        }
    }

    /// 重新载入CRL
    if (signalUser.reloadCRL)
    {
        msg = MW_RELOAD_CRL;
        for (k=0; k<NWorkerProcess; k++)
        {
            if (WorkerProcess[k].pid != -1)
                send(WorkerProcess[k].pipefd[0], &msg, 1, 0);
        }
    }

    MSG(LOG_INFO, "Master[%d] receives the user2 signal!", (int)getpid());
}

/**
  * @brief  创建新的worker_process
  * @param  worker进程索引
  * @retval master进程返回-1，worker进程返回0
  */
static int create_new_worker_process(int workerProcessIndex)
{
    int i;

    if (socketpair(PF_UNIX, SOCK_STREAM, 0, WorkerProcess[workerProcessIndex].pipefd) != 0)
    {
        GetLastErrorMsg(LOG_ERROR, "Failing to create a new worker process, because of");
        WorkerProcess[workerProcessIndex].pipefd[0] = -1;
        WorkerProcess[workerProcessIndex].pipefd[1] = -1;
        StopWorkingWorker[workerProcessIndex] = workerProcessIndex;
        alarm(MASTER_RECREATE_WORKER_MAXTIME);   // 创建worker失败，MASTER_RECREATE_WORKER_MAXTIME秒后重新创建
        return -1;
    }

    if ((WorkerProcess[workerProcessIndex].pid = fork()) < 0)
    {
        GetLastErrorMsg(LOG_ERROR, "Failing to create a new worker process, , because of");
        close(WorkerProcess[workerProcessIndex].pipefd[0]);
        close(WorkerProcess[workerProcessIndex].pipefd[1]);
        WorkerProcess[workerProcessIndex].pipefd[0] = -1;
        WorkerProcess[workerProcessIndex].pipefd[1] = -1;
        WorkerProcess[workerProcessIndex].pid = -1;
        StopWorkingWorker[workerProcessIndex] = workerProcessIndex;
        alarm(MASTER_RECREATE_WORKER_MAXTIME);   // 创建worker失败，MASTER_RECREATE_WORKER_MAXTIME秒后重新创建
        return -1;
    }

    /// master进程
    if (WorkerProcess[workerProcessIndex].pid > 0)
    {
        close(WorkerProcess[workerProcessIndex].pipefd[1]);
        WorkerProcess[workerProcessIndex].busyRatio = 0;
        BusyRatioPtr[workerProcessIndex] = ShmBusyRatio + workerProcessIndex * SERVER_SHM_CT_PER_SIZE;
        add_read_fd(Epollfd, WorkerProcess[workerProcessIndex].pipefd[0], ET);
        StopWorkingWorker[workerProcessIndex] = -1;
        return -1;
    }
    /// worker进程
    else
    {
        WorkerProcess[workerProcessIndex].busyRatio = 0;
        WorkerProcess[workerProcessIndex].pid = (int)getpid();
        BusyRatioPtr[workerProcessIndex] = ShmBusyRatio + workerProcessIndex * SERVER_SHM_CT_PER_SIZE;
        WorkerProcessIndex = workerProcessIndex;

        /* 禁止调用close_fd来关闭不必要的标识符(会将文件表中的文件描述符从Master进程中的Epollfd中删除)，否则程序会异常 */
        for (i=0; i<NWorkerProcess; i++)
            if (WorkerProcess[i].pid != -1)
                close(WorkerProcess[i].pipefd[0]);
        close(SigPipefd[1]);
        close(SigPipefd[0]);
        close(Epollfd);

        free(StopWorkingWorker);
        StopWorkingWorker = NULL;

        return 0;
    }
}

/**
  * @brief  执行master进程循环
  * @param  NULL
  * @retval master进程返回-1,worker进程返回进程组下标
  */
static int run_master_process_cycle()
{
    int     i, j, k, sum, stat, sockfd, number, retval;
    struct  epoll_event events[MASTER_MAX_EVENT_NUMBER];
    u_char  msg;
    u_char  sockBuf[MASTER_SOCK_BUFLEN];
    pid_t   workerPid;
    BOOL    stopMasterCycle = FALSE;
    BOOL    createNewWorkerProcess = TRUE;
    ZeroMemory(events, sizeof(events));

    init_master_signal_pipe();
    for (i=0; i<NWorkerProcess; i++)
        add_read_fd(Epollfd, WorkerProcess[i].pipefd[0], ET);
    add_read_fd(Epollfd, Listenfd, ET);

    MSG(LOG_INFO, "Master[%d] starts working cycle now!", (int)getpid());

    /// 主进程循环
    while (!stopMasterCycle)
    {
        number = epoll_wait(Epollfd, events, MASTER_MAX_EVENT_NUMBER, MASTER_EPOLL_WAIT_TIME);

        if ((number < 0) && (errno != EINTR))
        {
            GetLastSocketErrorMsg(LOG_ERROR, "master epoll_wait error msg:");
            /// 杀死所有worker进程，并且回收worker进程
            for (i=0; i<NWorkerProcess; i++)
                kill(WorkerProcess[i].pid, SIGTERM);
            for (i=0; i<MASTER_RECYCLE_WORKERS_MAXTIME; i++)
            {
                sleep(1);
                while ( (workerPid = waitpid(-1, &stat, WNOHANG)) > 0 )
                {
                    for (j=0; j<NWorkerProcess; j++)
                    {
                        if (WorkerProcess[j].pid == workerPid)
                            WorkerProcess[j].pid = -1;
                    }

                    for (j=0, sum=0; j<NWorkerProcess; j++)
                    {
                        if (WorkerProcess[j].pid == -1)
                            sum++;
                        if (sum == NWorkerProcess)
                        {
                            retval = MASTER_EXIT_ABNORMAL;
                            goto tag_finish;
                        }
                    }
                }
            }
            retval = MASTER_EXIT_FATAL;
            goto tag_finish;
        }

        if (number == 0)
        {
#ifdef ENABLE_DEBUG
            master_timer_timeout();
#endif
            continue;
        }

        for (i=0; i<number; i++)
        {
            sockfd = events[i].data.fd;

            /// 用户请求连接
            if ( (sockfd == Listenfd) && (events[i].events & EPOLLIN) )
            {
                msg = MW_SOCK_ACCEPT;
                j = get_most_free_service();
                send(WorkerProcess[j].pipefd[0], &msg, 1, 0); // 不关心发送成功与否
                //LOG("Send request to worker[%d]\n", j);
            }
            /// 信息管道
            else if( (sockfd == SigPipefd[0]) && (events[i].events & EPOLLIN) )
            {
                while (TRUE)
                {
                    retval = recv(sockfd, sockBuf, MASTER_SOCK_BUFLEN, 0);
                    if (retval <= 0)
                        break;

                    for (j=0; j<retval; j++)
                    {
                        switch (sockBuf[j])
                        {
                            case SIGCHLD :
                            {
                                while ( (workerPid = waitpid(-1, &stat, WNOHANG)) > 0 )
                                {
                                    for (k=0; k<NWorkerProcess; k++)
                                    {
                                        if (WorkerProcess[k].pid == workerPid)
                                        {
                                            MSG(LOG_INFO, "worker[%d-%d] has been joined!", k, WorkerProcess[k].pid);
                                            close_fd(Epollfd, WorkerProcess[k].pipefd[0]);
                                            WorkerProcess[k].busyRatio = 0;
                                            WorkerProcess[k].pipefd[0] = -1;
                                            WorkerProcess[k].pid = -1;
                                            /// 是否创建新的worker进程
                                            if (createNewWorkerProcess == TRUE)
                                                if (create_new_worker_process(k) == 0)
                                                    return k;
                                            break;
                                        }
                                    }

                                    /// 检查worker进程是否全部退出
                                    for (k=0, sum=0; k<NWorkerProcess; k++)
                                    {
                                        if (WorkerProcess[k].pid == -1)
                                            sum++;
                                        if (sum == NWorkerProcess)
                                            stopMasterCycle = TRUE;
                                    }
                                }

                                break; // break case
                            }

                            case SIGALRM :
                            {
                                for (k=0; k<NWorkerProcess; k++)
                                    if ( (createNewWorkerProcess == TRUE) && (StopWorkingWorker[k] >=0) )
                                        if (create_new_worker_process(k) == 0)
                                            return k;
                                break;
                            }

                            case SIGUSR2 :
                            {
                                handle_master_signal_user2();
                                break;
                            }

                            case SIGTERM :
                            case SIGINT :
                            {
                                createNewWorkerProcess = FALSE;
                                MSG(LOG_INFO, "Master[%d] will send quit signal to all workers!", (int)getpid());
                                for (k=0; k<NWorkerProcess; k++)
                                    kill(WorkerProcess[k].pid, SIGTERM);
                                break;
                            }

                            default :
                                break;
                        } // end of switch

                    } // end of for

                    if (retval < MASTER_SOCK_BUFLEN)
                        break;
                } // end of while
            }
            /// 读取来自子进程信息
            else if (events[i].events & EPOLLIN)
            {
                while (TRUE)
                {
                    retval = recv(sockfd, sockBuf, MASTER_SOCK_BUFLEN, 0);
                    if ((retval<MASTER_SOCK_BUFLEN) || (retval<=0) )
                        break;
                }

                for(j=0; j<NWorkerProcess; j++)
                {
                    if((WorkerProcess[j].pid != -1) && (sockfd == WorkerProcess[j].pipefd[0]) )
                    {
                        ShmBusyRatioLock();
                        WorkerProcess[j].busyRatio = atoi(BusyRatioPtr[j]);
                        ShmBusyRatioUnlock();
                        break;
                    }
                }
            }
            else
            {}

        } // end of for

    } // end of while

    retval = MASTER_EXIT_NORMAL;

tag_finish:
    /// 关闭所有描述符
    for(i=0; i<NWorkerProcess; i++)
    {
        if (WorkerProcess[i].pipefd[0] > 0)
            close_fd(Epollfd, WorkerProcess[i].pipefd[0]);
    }
    close(SigPipefd[1]);
    close_fd(Epollfd, SigPipefd[0]);
    if ((retval == MASTER_EXIT_NORMAL) || (retval == MASTER_EXIT_FATAL))
    {
        close_fd(Epollfd, Listenfd);
    }
    else
    {
        remove_fd(Epollfd, Listenfd);
    }
    close(Epollfd);

    MSG(LOG_INFO, "Master[%d] stops working %s now!", (int)getpid(),
                  (retval == MASTER_EXIT_NORMAL  ) ? "normally"   :
                 ((retval == MASTER_EXIT_ABNORMAL) ? "abnormally" : "fatally"));
    return retval;
}

/**
  * @brief  记录所有的worker进程id
  * @param  NULL
  * @retval void
  */
static void write_down_all_workers_pid()
{
    int     i;
    FILE   *fp;

    assert_param((fp = fopen(SERVER_BASEPATH SERVER_WORKER_PID, "wt")) != NULL );
    for (i=0; i<NWorkerProcess; i++)
        fprintf(fp, "%d\n", WorkerProcess[i].pid);

    fclose(fp);
}

/**
  * @brief  根据WorkerIndex值启动master和worker进程
  * @param  NULL
  * @retval void
  */
static void run_master_and_workers()
{
    int retval;

tag_restart:
    if (WorkerProcessIndex == -1)
    {
        write_down_all_workers_pid();
        retval = run_master_process_cycle();
        write_down_all_workers_pid();
        if (retval >= 0)
        {
            MSG(LOG_INFO, "An old worker cycle exited abnormally, so a new worker[%d] cycle will be run again!", (int)getpid());
            run_worker_process_cycle();
        }
        else if (retval == MASTER_EXIT_ABNORMAL)
        {
            MSG(LOG_INFO, "Master cycle exited abnormally, so master and the new workers cycle will be run again!");
            create_worker_process_pool();
            goto tag_restart;
        }
    }
    else
    {
        run_worker_process_cycle();
    }
}

/**
  * @brief  创建工作进程池
  * @param  NULL
  * @retval void
  */
static void create_worker_process_pool()
{
    int i, j;

    for (i=0; i<NWorkerProcess; i++)
    {
        assert_param(socketpair(PF_UNIX, SOCK_STREAM, 0, WorkerProcess[i].pipefd) == 0);
        WorkerProcess[i].pid = fork();
        assert_param(WorkerProcess[i].pid >= 0);

        /// master进程
        if (WorkerProcess[i].pid > 0)
        {
            close(WorkerProcess[i].pipefd[1]);
            WorkerProcess[i].busyRatio = 0;
            BusyRatioPtr[i] = ShmBusyRatio + i * SERVER_SHM_CT_PER_SIZE;
            continue;
        }
        /// worker进程
        else
        {
            for (j=0; j<=i; j++)
                close(WorkerProcess[j].pipefd[0]);
            WorkerProcess[i].busyRatio = 0;
            WorkerProcess[i].pid = (int)getpid();
            BusyRatioPtr[i] = ShmBusyRatio + i * SERVER_SHM_CT_PER_SIZE;
            WorkerProcessIndex = i;

            free(StopWorkingWorker);
            StopWorkingWorker = NULL;
            return;
        }
    }

    MSG(LOG_INFO, "Master[%d] inits the worker pool successfully!", (int)getpid());
}

/**
  * @brief  创建共享内存(当修改共享内存时，前16用当前时间填充,标记唯一性,默认填充“1234567890ABCDEF”)
  * @param  argv1: flag
  * @retval NULL
  */
static void create_shared_memory(int flag)
{
    key_t shmLogoffKey;
    key_t shmBusyRatioKey;

    /// 创建心跳包共享内存
    assert_param((shmLogoffKey = ftok(SERVER_KEY_PATH, 'd')) != -1);
    if ((ShmLogoffShmId = shmget(shmLogoffKey, SERVER_SHM_LF_SIZE, IPC_CREAT|IPC_EXCL|0600)) == -1)
        ShmLogoffShmId = shmget(shmLogoffKey, SERVER_SHM_LF_SIZE, 0600);
    assert_param((ShmLogoff = (char *)shmat(ShmLogoffShmId, NULL, 0)) != (char *)-1);
    if (flag)
        ZeroMemory(ShmLogoff, SERVER_SHM_LF_SIZE);
    else
        return;

    /// 创建连接数共享内存
    assert_param((shmBusyRatioKey = ftok(SERVER_KEY_PATH, 'e')) != -1);
    if ((ShmBusyRatioShmId = shmget(shmBusyRatioKey, SERVER_SHM_CT_SIZE, IPC_CREAT|IPC_EXCL|0600)) == -1)
        ShmBusyRatioShmId = shmget(shmBusyRatioKey, SERVER_SHM_CT_SIZE, 0600);
    assert_param((ShmBusyRatio = (char *)shmat(ShmBusyRatioShmId, NULL, 0)) != (char *)-1);
    ZeroMemory(ShmBusyRatio, SERVER_SHM_CT_SIZE);

    MSG(LOG_INFO, "Master[%d] inits all shared memory successfully!", (int)getpid());
}

/**
  * @brief  记录master pid
  * @param  NULL
  * @retval NULL
  */
static void write_down_master_process_pid()
{
    FILE *fp = NULL;
    assert_param((fp = fopen(SERVER_PID_PATH, "wt")) != NULL);
    fprintf(fp, "%d", getpid());
    fclose(fp);
}

/**
  * @brief  初始化所有数据
  * @param  NULL
  * @retval NULL
  */
static void init_all_data()
{
    int i;

    /// 日志处理
    close_log();
    clean_log();
    open_log();

    /// 配置参数合法性检查
    if (SERVER_PORT < 2000)
        MSG(LOG_WARNING, "Server port shall great than 2000");
    if (inet_addr(SERVER_IP) == INADDR_NONE)
        MSG(LOG_FATAL, "Server IP is invalid");
    if (NWorkerProcess                <= 0  || NWorkerProcess > WORKER_MAX_NPROCESS)
        NWorkerProcess                 = 4;
    if (WORKER_LOGIN_TIMEOUT           < 2  || WORKER_LOGIN_TIMEOUT           > 5   )
        WORKER_LOGIN_TIMEOUT           = 3;
    if (WORKER_HEARTBEAT_TIMEOUT       < 30 || WORKER_HEARTBEAT_TIMEOUT       > 600 )
        WORKER_HEARTBEAT_TIMEOUT       = 30;
    if (MASTER_RECYCLE_WORKERS_MAXTIME < 60 || MASTER_RECYCLE_WORKERS_MAXTIME > 600 )
        MASTER_RECYCLE_WORKERS_MAXTIME = 300;
    if (MASTER_RECREATE_WORKER_MAXTIME < 30 || MASTER_RECREATE_WORKER_MAXTIME > 300 )
        MASTER_RECYCLE_WORKERS_MAXTIME = 60;
    if (ServerConfig.databaseTimeout   < 50 || ServerConfig.databaseTimeout   > 1000)
        ServerConfig.databaseTimeout   = 50;
    SERVER_DAEMON = BoolCast(SERVER_DAEMON);

    /// 进程索引，用于区分启动master和worker进程
    WorkerProcessIndex = -1;

    /// 创建两把进程锁
    ShmLogoffSemId  = sem_process_init(SERVER_KEY_PATH, 'b');
    assert_param(ShmLogoffSemId != -1);
    ShmBusyRatioSemId = sem_process_init(SERVER_KEY_PATH, 'c');
    assert_param(ShmBusyRatioSemId != -1);

    /// 根据进程数目创建worker进程对象
    WorkerProcess = (WORKER_PROCESS_T *)calloc(NWorkerProcess, sizeof(WORKER_PROCESS_T));
    assert_param(WorkerProcess);

    /// 为每个worker进程划分共享内存
    BusyRatioPtr = (char **)calloc(NWorkerProcess, sizeof(char *));
    assert_param(BusyRatioPtr);

    /// 用于启动意外终止的worker进程
    StopWorkingWorker = (int *)calloc(NWorkerProcess, sizeof(int));
    assert_param(StopWorkingWorker != NULL);
    for (i=0; i<NWorkerProcess; i++)
        StopWorkingWorker[i] = -1;

    MSG(LOG_INFO, "Master[%d] inits all data of server framework successfully!", (int)getpid());
}

/**
  * @brief  启动master程序
  * @param  NULL
  * @retval NULL
  */
static void start_master_process()
{
    /// 程序后台化
    if (SERVER_DAEMON)
        assert_param(daemonize() == 0);

    /// 初始化所有数据
    init_all_data();

    /// 创建共享内存
    create_shared_memory(1);

    /// 创建监听套接字
    Listenfd = tcp_listen(SERVER_PORT, SERVER_IP);

    /// 记录当前进程号
    write_down_master_process_pid();

    /// 创建worker process pool
    create_worker_process_pool();

    /// 启动master-workers
    run_master_and_workers();

    /// 退出数据清理
    cleanup_before_master_exit();
}

/**
  * @brief  发送剔除用户下线的信号
  * @param  arg1;aa;bb;cc;
  * @retval 成功返回0,失败返回其它
  */
static int send_remove_signal(const char *content)
{
    int   i, len, sum;
    char  buffer[6] = {0};
    FILE *fp = NULL;

    /// 检查参数合法(检查通过也不能说明参数绝对合法)
    if (content == NULL || strlen(content) < 4)
    {
        MSG(LOG_ERROR, "Argument invalid, you should type like this: -remove aa" SPLIT "bb" SPLIT "cc" SPLIT);
        return 1;
    }
    for (i=0, sum=0, len = strlen(content); i<len; i++)
    {
        if (content[i] == SPLIT[0])
            sum++;
    }
    if (sum == 0)
    {
        MSG(LOG_ERROR, "Argument unmatch, you should type like this: -remove aa" SPLIT "bb" SPLIT "cc" SPLIT);
        return 2;
    }

    /// 获取master进程号
    if ( (fp = fopen(SERVER_PID_PATH, "rt")) == NULL)
    {
        GetLastErrorMsg(LOG_ERROR, "Failing to open with read the master pid file, because of");
        return 3;
    }
    fgets(buffer, sizeof(buffer), fp);
    fclose(fp);

    /// 打开共享内存
    create_shared_memory(0);

    /// 操作共享内存
    snprintf(ShmLogoffBody, SERVER_SHM_LF_BODY_SIZE - 1, "%s", content);

    /// 发送消息master进程
    assert_param(kill(atoi(buffer), SIGUSR2) == 0);

    /// 检查worker进程处理数否完毕，3秒后仍然未退出，强制置0
    for (i=0; i<3; i++)
    {
        sleep(1);
        if (atoi(ShmLogoffHead) <= 0)
        {
            MSG(LOG_INFO, "All workers handle shared memory task in given time!");
            return 0;
        }
    }
    MSG(LOG_INFO, "Some worker can not handle shared memory task in given time!");
    snprintf(ShmLogoffHead, SERVER_SHM_LF_HEAD_SIZE - 1, "%d", 0);

    return 0;
}

/**
  * @brief  发送通用信号
  * @param  NULL
  * @retval NULL
  */
static void send_general_signal()
{
    char  buffer[6] = {0};
    FILE *fp = NULL;

    /// 获取master进程号
    assert_param((fp=fopen(SERVER_PID_PATH, "rt")) != NULL);
    fgets(buffer, sizeof(buffer), fp);
    fclose(fp);

    /// 发送消息master进程
    assert_param(kill(atoi(buffer), SIGUSR2) == 0);
}

/**
  * @brief  关闭master程序
  * @param  NULL
  * @retval void
  */
static void stop_master_process()
{
    FILE *fp = NULL;
    char  buffer[6] = {0};

    assert_param((fp=fopen(SERVER_PID_PATH, "rt")) != NULL);
    fgets(buffer, sizeof(buffer), fp);
    fclose(fp);

    /// 终止master进程，并且捕捉该信号
    if (kill(atoi(buffer), SIGTERM) < 0)
    {
        GetLastErrorMsg(LOG_ERROR, "Fail to shut down the " SERVER_NAME ", because of");
        sleep(3);
        kill(atoi(buffer), SIGKILL);
    }
    else
    {
        MSG(LOG_INFO, "Shuting down the " SERVER_NAME " successfully!");
    }

    close_log();

    /// 移除master进程号文件
    remove(SERVER_PID_PATH);
}

/**
  * @brief  打印帮助
  * @param  NULL
  * @retval void
  */
static void show_help()
{
    ConsoleOutput(SERVER_NAME " Version: " SERVER_VERSION   CR
        "usage: " SERVER_NAME " [sSvh] [start, stop, reconfig, ca, crl, help]"
                              " [-config configuration path]"
                              " [-remove aaa;bbb;ccc;]"     CR
                                                            CR
        "Options:"                                          CR
        "  -start     : run  the " SERVER_NAME              CR
        "  -stop      : quit the " SERVER_NAME              CR
        "  -config    : set  the " SERVER_NAME " configuration file path" CR
        "  -reconfig  : reread configuration file!"         CR
        "  -ca        : reload certificate authority"       CR
        "  -crl       : reload certificate revoked list"    CR
        "  -remove    : make user logoff"                   CR
        "  -version   : show the version of " SERVER_NAME   CR
        "  -help      : help about " SERVER_NAME            CR);
}

/**
  * @brief  版本号输出
  * @param  NULL
  * @retval void
  */
static void show_version()
{
    ConsoleOutput("%s version: %s" CR, SERVER_NAME, SERVER_VERSION);
}

/**
  * @brief  写下信号配置参数
  * @param  arg1:signalUser
  * @retval void
  */
void write_down_signal_data(SIGNAL_USER_T *signalUser)
{
    FILE *fp;

    if (access(SERVER_BASEPATH, 0) != 0)
        assert_param(mkdir(SERVER_BASEPATH, 0755) == 0);
    assert_param((fp = fopen(SERVER_BASEPATH SERVER_SIGNAL_PATH, "wb")) != NULL);
    fwrite(signalUser, sizeof(SIGNAL_USER_T), 1, fp);

    fclose(fp);
}

/**
  * @brief  解析命令行参数
  * @param  arg1:arc  arg2:argv  arg3:commandParam  arg4:signalUser
  * @retval void
  */
static void
parse_server_command(int argc, char *argv[], COMMAND_PARAM_T *commandParam, SIGNAL_USER_T *signalUser)
{
    struct option serverOptions[] =
               {{"start",      no_argument,       NULL, OPT_START      },
                {"stop",       no_argument,       NULL, OPT_STOP       },
                {"configpath", required_argument, NULL, OPT_CONFIG_PATH},
                {"reconfig",   no_argument,       NULL, OPT_RECONFIG   },
                {"remove",     required_argument, NULL, OPT_REMOVE     },
                {"ca",         no_argument,       NULL, OPT_RELOAD_CA  },
                {"crl",        no_argument,       NULL, OPT_RELOAD_CRL },
                {"version",    no_argument,       NULL, OPT_VERSION    },
                {"help",       no_argument,       NULL, OPT_HELP       },
                {NULL,         0,                 NULL, 0              }};

    ZeroMemory(signalUser,   sizeof(SIGNAL_USER_T));
    ZeroMemory(commandParam, sizeof(COMMAND_PARAM_T));

    while (TRUE)
    {
        switch(getopt_long_only(argc, argv, "sSvh", serverOptions, NULL))
        {
            case 's' :
            case OPT_START :
                commandParam->start = 1;
                break;

            case 'S' :
            case OPT_STOP :
                commandParam->stop = 1;
                break;

            case OPT_CONFIG_PATH :
                SERVER_CNF_PATH = optarg;
                if (access(SERVER_CNF_PATH, 0) != 0)
                    MSG(LOG_FATAL, "The config file specified by manager can not access!");
                break;

            case OPT_RECONFIG :
                commandParam->reconfig = 1;
                signalUser->reconfig   = 1;
                break;

            case OPT_REMOVE :
                commandParam->remove   = 1;
                signalUser->remove     = 1;
                commandParam->remmsg   = optarg;
                break;

            case OPT_RELOAD_CA :
                commandParam->reloadCA = 1;
                signalUser->reloadCA   = 1;
                break;

            case OPT_RELOAD_CRL :
                commandParam->reloadCRL = 1;
                signalUser->reloadCRL   = 1;
                break;

            case 'v' :
            case OPT_VERSION :
                commandParam->version = 1;
                break;

            case 'h' :
            case OPT_HELP :
                commandParam->help = 1;
                break;

            case -1:
            default :
                write_down_signal_data(signalUser);
                return;
        }
    }
}

/**
  * @brief  服务器主程序
  * @param  arg1:arc  arg2:argv
  * @retval 0
  */
int server_main(int argc, char *argv[])
{
    int             retval = 0;
    SIGNAL_USER_T   signalUser;
    COMMAND_PARAM_T commandParam;
    /// 解析命令行参数
    parse_server_command(argc, argv, &commandParam, &signalUser);
    /// 获取配置文件信息
    get_server_config();
    get_rule_config(ServerConfig.rulePath);
    get_stunnel_config(ServerConfig.stunnelPath);
    get_utunnel_config(ServerConfig.utunnelPath);
    get_openvpn_config(ServerConfig.openvpnPath);
    ///****************patched by xiaohu mod***********2016/03/17*****fsadfasf**
    ///note:由于配置文件的原因，以下分别采取了不同办法来处理，参照宏定义的说明
    get_check_run_win_config(CHECK_RUN_WIN_PATH);   //得到checkrunningswitch.conf
    get_check_pre_and_config(CHECK_PRE_AND_PATH);   //adcheckswitch.conf
    get_check_run_and_config(CHECK_RUN_AND_PATH);   //adcheckrunningswitch.conf

    get_check_configs(CHECK_PRE_PROC_WIN_PATH,check_pre_proc_win_buf);
    get_check_configs(CHECK_PRE_TYPE_AND_PATH,check_pre_type_and_buf);
    get_check_configs(CHECK_RUN_PROC_WIN_PATH,check_run_proc_win_buf);

    if(filter_element_pre( SERVER_ANDROID_DOOR_CNF) == 0)
    {
        PreAndroidTypeDoor = 0;
    }
    ///********************************************2016/03/17*********fsadfasf**

    /// 获取服务器版本号
    if (commandParam.version)
    {
        show_version();
    }
    /// 显示帮助
    else if (commandParam.help)
    {
        show_help();
    }
    /// 关闭程序
    else if (commandParam.stop)
    {
        stop_master_process();
    }
    /// 发送通用信号
    else if (commandParam.reconfig || commandParam.reloadCA || commandParam.reloadCRL)
    {
        send_general_signal();
    }
    /// 发送剔除用户下线的信号
    else if (commandParam.remove)
    {
        retval = send_remove_signal(commandParam.remmsg);
    }
    /// 启动程序
    else if (commandParam.start)
    {
        start_master_process();
    }
    else
    {
        show_help();
    }

    /// 释放配置文件
    free_server_config();
    free_stunnel_config();
    free_utunnel_config();
    free_openvpn_config();

    return retval;
}

/******************* (C) COPYRIGHT 2016 上海今中网络科技有限公司 *****END OF FILE****/

